{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import re\n",
    "import sys\n",
    "import itertools\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import random\n",
    "import os\n",
    "import pickle\n",
    "import codecs\n",
    "from gensim import corpora\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labels = pd.read_csv('data/label.csv', names =['label'])['label'].tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9584"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42692"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('data/processed_trumptwitterarchive.txt', 'r') as data_file:\n",
    "    json_data = data_file.read()\n",
    "data = json.loads(json_data)\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = pd.read_csv('data/sentimentAnalysis/senti.csv')\n",
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df['time'] = sent['time'].apply(lambda x: x[:-6])\n",
    "df['texts'] = sent['texts']\n",
    "df['retweet_count'] = sent['retweet_count']\n",
    "df['favorite_count'] = sent['favorite_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>texts</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-11-16 15:18:27</td>\n",
       "      <td>dow hit first time ever highest ever gee pelos...</td>\n",
       "      <td>4301</td>\n",
       "      <td>15011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-11-16 13:56:23</td>\n",
       "      <td>rt armyst lt clint lorance one two u army offi...</td>\n",
       "      <td>9149</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-11-16 13:51:34</td>\n",
       "      <td>louisiana vote today great governor</td>\n",
       "      <td>7311</td>\n",
       "      <td>24555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-11-16 13:08:14</td>\n",
       "      <td>good morning louisiana poll open atam get vote...</td>\n",
       "      <td>10699</td>\n",
       "      <td>30751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-11-16 06:03:27</td>\n",
       "      <td>rt big day louisiana tomorrow get vote replace...</td>\n",
       "      <td>10084</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  time                                              texts  \\\n",
       "0  2019-11-16 15:18:27  dow hit first time ever highest ever gee pelos...   \n",
       "1  2019-11-16 13:56:23  rt armyst lt clint lorance one two u army offi...   \n",
       "2  2019-11-16 13:51:34                louisiana vote today great governor   \n",
       "3  2019-11-16 13:08:14  good morning louisiana poll open atam get vote...   \n",
       "4  2019-11-16 06:03:27  rt big day louisiana tomorrow get vote replace...   \n",
       "\n",
       "   retweet_count  favorite_count  \n",
       "0           4301           15011  \n",
       "1           9149               0  \n",
       "2           7311           24555  \n",
       "3          10699           30751  \n",
       "4          10084               0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['begin_time'] = df['time'].apply(lambda x: x[:-2]+'00')\n",
    "df['begin_time'] = df['begin_time'].apply(lambda x: pd.to_datetime(x))\n",
    "df['end_time'] = df['begin_time'] + timedelta(minutes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>texts</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>begin_time</th>\n",
       "      <th>end_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-11-16 15:18:27</td>\n",
       "      <td>dow hit first time ever highest ever gee pelos...</td>\n",
       "      <td>4301</td>\n",
       "      <td>15011</td>\n",
       "      <td>2019-11-16 15:18:00</td>\n",
       "      <td>2019-11-16 15:23:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-11-16 13:56:23</td>\n",
       "      <td>rt armyst lt clint lorance one two u army offi...</td>\n",
       "      <td>9149</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-11-16 13:56:00</td>\n",
       "      <td>2019-11-16 14:01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-11-16 13:51:34</td>\n",
       "      <td>louisiana vote today great governor</td>\n",
       "      <td>7311</td>\n",
       "      <td>24555</td>\n",
       "      <td>2019-11-16 13:51:00</td>\n",
       "      <td>2019-11-16 13:56:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-11-16 13:08:14</td>\n",
       "      <td>good morning louisiana poll open atam get vote...</td>\n",
       "      <td>10699</td>\n",
       "      <td>30751</td>\n",
       "      <td>2019-11-16 13:08:00</td>\n",
       "      <td>2019-11-16 13:13:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-11-16 06:03:27</td>\n",
       "      <td>rt big day louisiana tomorrow get vote replace...</td>\n",
       "      <td>10084</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-11-16 06:03:00</td>\n",
       "      <td>2019-11-16 06:08:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  time                                              texts  \\\n",
       "0  2019-11-16 15:18:27  dow hit first time ever highest ever gee pelos...   \n",
       "1  2019-11-16 13:56:23  rt armyst lt clint lorance one two u army offi...   \n",
       "2  2019-11-16 13:51:34                louisiana vote today great governor   \n",
       "3  2019-11-16 13:08:14  good morning louisiana poll open atam get vote...   \n",
       "4  2019-11-16 06:03:27  rt big day louisiana tomorrow get vote replace...   \n",
       "\n",
       "   retweet_count  favorite_count          begin_time            end_time  \n",
       "0           4301           15011 2019-11-16 15:18:00 2019-11-16 15:23:00  \n",
       "1           9149               0 2019-11-16 13:56:00 2019-11-16 14:01:00  \n",
       "2           7311           24555 2019-11-16 13:51:00 2019-11-16 13:56:00  \n",
       "3          10699           30751 2019-11-16 13:08:00 2019-11-16 13:13:00  \n",
       "4          10084               0 2019-11-16 06:03:00 2019-11-16 06:08:00  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_econ = pd.read_csv('data/all_cleaned_SPX_data.csv')\n",
    "data_econ['datetime'] = pd.to_datetime(data_econ['datetime'])\n",
    "df_econ = data_econ[['datetime', 'close']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>texts</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>begin_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>close_x</th>\n",
       "      <th>close_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-11-07 15:43:29</td>\n",
       "      <td>stock market big today new record enjoy</td>\n",
       "      <td>20211</td>\n",
       "      <td>114187</td>\n",
       "      <td>2019-11-07 15:43:00</td>\n",
       "      <td>2019-11-07 15:48:00</td>\n",
       "      <td>3080.94</td>\n",
       "      <td>3082.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-11-07 15:41:53</td>\n",
       "      <td>radical left dems lamestream medium trying mak...</td>\n",
       "      <td>18328</td>\n",
       "      <td>72619</td>\n",
       "      <td>2019-11-07 15:41:00</td>\n",
       "      <td>2019-11-07 15:46:00</td>\n",
       "      <td>3082.59</td>\n",
       "      <td>3081.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-11-07 15:27:57</td>\n",
       "      <td>amazon washington post three lowlife reporter ...</td>\n",
       "      <td>24007</td>\n",
       "      <td>83333</td>\n",
       "      <td>2019-11-07 15:27:00</td>\n",
       "      <td>2019-11-07 15:32:00</td>\n",
       "      <td>3082.41</td>\n",
       "      <td>3083.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-11-07 15:18:53</td>\n",
       "      <td>thank #MAGA</td>\n",
       "      <td>14962</td>\n",
       "      <td>65540</td>\n",
       "      <td>2019-11-07 15:18:00</td>\n",
       "      <td>2019-11-07 15:23:00</td>\n",
       "      <td>3082.46</td>\n",
       "      <td>3082.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-11-07 15:16:15</td>\n",
       "      <td>explained next week fake hearing trial house i...</td>\n",
       "      <td>35482</td>\n",
       "      <td>132078</td>\n",
       "      <td>2019-11-07 15:16:00</td>\n",
       "      <td>2019-11-07 15:21:00</td>\n",
       "      <td>3084.20</td>\n",
       "      <td>3080.28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  time                                              texts  \\\n",
       "0  2019-11-07 15:43:29            stock market big today new record enjoy   \n",
       "1  2019-11-07 15:41:53  radical left dems lamestream medium trying mak...   \n",
       "2  2019-11-07 15:27:57  amazon washington post three lowlife reporter ...   \n",
       "3  2019-11-07 15:18:53                                        thank #MAGA   \n",
       "4  2019-11-07 15:16:15  explained next week fake hearing trial house i...   \n",
       "\n",
       "   retweet_count  favorite_count          begin_time            end_time  \\\n",
       "0          20211          114187 2019-11-07 15:43:00 2019-11-07 15:48:00   \n",
       "1          18328           72619 2019-11-07 15:41:00 2019-11-07 15:46:00   \n",
       "2          24007           83333 2019-11-07 15:27:00 2019-11-07 15:32:00   \n",
       "3          14962           65540 2019-11-07 15:18:00 2019-11-07 15:23:00   \n",
       "4          35482          132078 2019-11-07 15:16:00 2019-11-07 15:21:00   \n",
       "\n",
       "   close_x  close_y  \n",
       "0  3080.94  3082.54  \n",
       "1  3082.59  3081.32  \n",
       "2  3082.41  3083.63  \n",
       "3  3082.46  3082.57  \n",
       "4  3084.20  3080.28  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merge1 = pd.merge(df, df_econ, left_on = 'begin_time', right_on='datetime').drop(['datetime'], axis=1)\n",
    "df_merge2 = pd.merge(df_merge1, df_econ, left_on = 'end_time', right_on='datetime').drop(['datetime'], axis=1)\n",
    "df_merge2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9312"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_merge2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merge2['label'] = (df_merge2['close_y'] > df_merge2['close_x'])*1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>texts</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>begin_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>close_x</th>\n",
       "      <th>close_y</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-11-07 15:43:29</td>\n",
       "      <td>stock market big today new record enjoy</td>\n",
       "      <td>20211</td>\n",
       "      <td>114187</td>\n",
       "      <td>2019-11-07 15:43:00</td>\n",
       "      <td>2019-11-07 15:48:00</td>\n",
       "      <td>3080.94</td>\n",
       "      <td>3082.54</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-11-07 15:41:53</td>\n",
       "      <td>radical left dems lamestream medium trying mak...</td>\n",
       "      <td>18328</td>\n",
       "      <td>72619</td>\n",
       "      <td>2019-11-07 15:41:00</td>\n",
       "      <td>2019-11-07 15:46:00</td>\n",
       "      <td>3082.59</td>\n",
       "      <td>3081.32</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-11-07 15:27:57</td>\n",
       "      <td>amazon washington post three lowlife reporter ...</td>\n",
       "      <td>24007</td>\n",
       "      <td>83333</td>\n",
       "      <td>2019-11-07 15:27:00</td>\n",
       "      <td>2019-11-07 15:32:00</td>\n",
       "      <td>3082.41</td>\n",
       "      <td>3083.63</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-11-07 15:18:53</td>\n",
       "      <td>thank #MAGA</td>\n",
       "      <td>14962</td>\n",
       "      <td>65540</td>\n",
       "      <td>2019-11-07 15:18:00</td>\n",
       "      <td>2019-11-07 15:23:00</td>\n",
       "      <td>3082.46</td>\n",
       "      <td>3082.57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-11-07 15:16:15</td>\n",
       "      <td>explained next week fake hearing trial house i...</td>\n",
       "      <td>35482</td>\n",
       "      <td>132078</td>\n",
       "      <td>2019-11-07 15:16:00</td>\n",
       "      <td>2019-11-07 15:21:00</td>\n",
       "      <td>3084.20</td>\n",
       "      <td>3080.28</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  time                                              texts  \\\n",
       "0  2019-11-07 15:43:29            stock market big today new record enjoy   \n",
       "1  2019-11-07 15:41:53  radical left dems lamestream medium trying mak...   \n",
       "2  2019-11-07 15:27:57  amazon washington post three lowlife reporter ...   \n",
       "3  2019-11-07 15:18:53                                        thank #MAGA   \n",
       "4  2019-11-07 15:16:15  explained next week fake hearing trial house i...   \n",
       "\n",
       "   retweet_count  favorite_count          begin_time            end_time  \\\n",
       "0          20211          114187 2019-11-07 15:43:00 2019-11-07 15:48:00   \n",
       "1          18328           72619 2019-11-07 15:41:00 2019-11-07 15:46:00   \n",
       "2          24007           83333 2019-11-07 15:27:00 2019-11-07 15:32:00   \n",
       "3          14962           65540 2019-11-07 15:18:00 2019-11-07 15:23:00   \n",
       "4          35482          132078 2019-11-07 15:16:00 2019-11-07 15:21:00   \n",
       "\n",
       "   close_x  close_y  label  \n",
       "0  3080.94  3082.54      1  \n",
       "1  3082.59  3081.32      0  \n",
       "2  3082.41  3083.63      1  \n",
       "3  3082.46  3082.57      1  \n",
       "4  3084.20  3080.28      0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merge2.t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
